{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 浏览器的控制库\n",
    "import webbrowser\n",
    "\n",
    "## 将抓取的网页保存到tmp.html，并打开一个新的标签窗口展示tmp.html\n",
    "def requests_view(response):\n",
    "    request_url = response.url\n",
    "    base_url = '<head><base href=\"%s\">'% (request_url)\n",
    "    base_url = base_url.encode()\n",
    "    content = response.content.replace(b\"<head>\", base_url)\n",
    "    tem_html = open('tmp.html', 'wb')\n",
    "    tem_html.write(content)\n",
    "    tem_html.close()\n",
    "    webbrowser.open_new_tab('tmp.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "url = 'http://www.spbeen.com/tool/request_info/'\n",
    "js_url = 'https://www.jianshu.com/p/ef4a9fdceb61'\n",
    "headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "{'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "resp = requests.get(js_url, headers=headers)\n",
    "print(resp)\n",
    "print(resp.request.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [403]>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 403 页面\n",
    "requests_view(resp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "reso = requests.get(js_url, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [403]>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 切换User-Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import faker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "fake = faker.Faker()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0; rv:1.9.4.20) Gecko/2018-02-16 05:20:25 Firefox/5.0\n",
      "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.2 (KHTML, like Gecko) Chrome/14.0.888.0 Safari/536.2\n",
      "Mozilla/5.0 (Macintosh; PPC Mac OS X 10_5_9; rv:1.9.2.20) Gecko/2012-12-01 12:40:41 Firefox/13.0\n",
      "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_9_8; rv:1.9.6.20) Gecko/2012-05-28 21:08:17 Firefox/3.8\n",
      "Mozilla/5.0 (Windows 95; wo-SN; rv:1.9.2.20) Gecko/2016-11-30 02:06:15 Firefox/3.6.13\n",
      "Mozilla/5.0 (X11; Linux x86_64; rv:1.9.7.20) Gecko/2019-01-14 21:13:46 Firefox/3.8\n",
      "Opera/8.25.(Windows 98; Win 9x 4.90; is-IS) Presto/2.9.161 Version/12.00\n",
      "Mozilla/5.0 (X11; Linux i686; rv:1.9.5.20) Gecko/2010-05-18 10:31:08 Firefox/3.8\n",
      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.0 (KHTML, like Gecko) Chrome/19.0.817.0 Safari/534.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, 10):\n",
    "    print(fake.user_agent())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 批量请求和提取User-Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mozilla/5.0 (Macintosh; PPC Mac OS X 10_9_0; rv:1.9.6.20) Gecko/2016-04-02 06:22:55 Firefox/3.8\n",
      "Opera/8.48.(X11; Linux x86_64; sc-IT) Presto/2.9.168 Version/12.00\n",
      "Mozilla/5.0 (Android 3.2.5; Mobile; rv:28.0) Gecko/28.0 Firefox/28.0\n",
      "Opera/9.62.(X11; Linux x86_64; bo-IN) Presto/2.9.187 Version/10.00\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "import time\n",
    "\n",
    "for i in range(1, 5):\n",
    "    response = requests.get(url, headers={'User-Agent':fake.user_agent()})\n",
    "    requests_view(response)\n",
    "    html = etree.HTML(response.text)\n",
    "    print(html.xpath('normalize-space(.//div[contains(text(),\"User-Agent\")]/../div[2]/text())'))\n",
    "#     print(html.xpath('normalize-space(.//div[contains(text(),\"User-Agent\")]/../div[2]/text())'))\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
